


<!DOCTYPE html>
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
<head>
  <meta charset="utf-8">
  
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  
  <title>torch.autograd.function &mdash; PyTorch master documentation</title>
  

  
  
  
  
    <link rel="canonical" href="https://pytorch.org/docs/stable/_modules/torch/autograd/function.html"/>
  

  

  
  
    

  

  <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
  <!-- <link rel="stylesheet" href="../../../_static/pygments.css" type="text/css" /> -->
  <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.10.0-beta/dist/katex.min.css" type="text/css" />
  <link rel="stylesheet" href="../../../_static/css/jit.css" type="text/css" />
  <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.11.1/dist/katex.min.css" type="text/css" />
  <link rel="stylesheet" href="../../../_static/katex-math.css" type="text/css" />
    <link rel="index" title="Index" href="../../../genindex.html" />
    <link rel="search" title="Search" href="../../../search.html" /> 

  
  <script src="../../../_static/js/modernizr.min.js"></script>

  <!-- Preload the theme fonts -->

<link rel="preload" href="../../../_static/fonts/FreightSans/freight-sans-book.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../../../_static/fonts/FreightSans/freight-sans-medium.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../../../_static/fonts/IBMPlexMono/IBMPlexMono-Medium.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../../../_static/fonts/FreightSans/freight-sans-bold.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../../../_static/fonts/FreightSans/freight-sans-medium-italic.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../../../_static/fonts/IBMPlexMono/IBMPlexMono-SemiBold.woff2" as="font" type="font/woff2" crossorigin="anonymous">

<!-- Preload the katex fonts -->

<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Math-Italic.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Main-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Main-Bold.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Size1-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Size4-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Size2-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Size3-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Caligraphic-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
</head>

<div class="container-fluid header-holder tutorials-header" id="header-holder">
  <div class="container">
    <div class="header-container">
      <a class="header-logo" href="https://pytorch.org/" aria-label="PyTorch"></a>

      <div class="main-menu">
        <ul>
          <li>
            <a href="https://pytorch.org/get-started">Get Started</a>
          </li>

          <li>
            <div class="ecosystem-dropdown">
              <a id="dropdownMenuButton" data-toggle="ecosystem-dropdown">
                Ecosystem
              </a>
              <div class="ecosystem-dropdown-menu">
                <a class="nav-dropdown-item" href="https://pytorch.org/hub"">
                  <span class=dropdown-title>Models (Beta)</span>
                  <p>Discover, publish, and reuse pre-trained models</p>
                </a>
                <a class="nav-dropdown-item" href="https://pytorch.org/ecosystem">
                  <span class=dropdown-title>Tools & Libraries</span>
                  <p>Explore the ecosystem of tools and libraries</p>
                </a>
              </div>
            </div>
          </li>

          <li>
            <a href="https://pytorch.org/mobile">Mobile</a>
          </li>

          <li>
            <a href="https://pytorch.org/blog/">Blog</a>
          </li>

          <li>
            <a href="https://pytorch.org/tutorials">Tutorials</a>
          </li>

          <li class="active">
            <a href="https://pytorch.org/docs/stable/index.html">Docs</a>
          </li>

          <li>
            <div class="resources-dropdown">
              <a id="resourcesDropdownButton" data-toggle="resources-dropdown">
                Resources
              </a>
              <div class="resources-dropdown-menu">
                <a class="nav-dropdown-item" href="https://pytorch.org/resources"">
                  <span class=dropdown-title>Developer Resources</span>
                  <p>Find resources and get questions answered</p>
                </a>
                <a class="nav-dropdown-item" href="https://pytorch.org/features">
                  <span class=dropdown-title>About</span>
                  <p>Learn about PyTorch’s features and capabilities</p>
                </a>
              </div>
            </div>
          </li>

          <li>
            <a href="https://github.com/pytorch/pytorch">Github</a>
          </li>
        </ul>
      </div>

      <a class="main-menu-open-button" href="#" data-behavior="open-mobile-menu"></a>
    </div>

  </div>
</div>


<body class="pytorch-body">

   

    

    <div class="table-of-contents-link-wrapper">
      <span>Table of Contents</span>
      <a href="#" class="toggle-table-of-contents" data-behavior="toggle-table-of-contents"></a>
    </div>

    <nav data-toggle="wy-nav-shift" class="pytorch-left-menu" id="pytorch-left-menu">
      <div class="pytorch-side-scroll">
        <div class="pytorch-menu pytorch-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
          <div class="pytorch-left-menu-search">
            

            
              
              
                <div class="version">
                  master (1.5.0 )
                </div>
              
            

            


  


<div role="search">
  <form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
    <input type="text" name="q" placeholder="Search Docs" />
    <input type="hidden" name="check_keywords" value="yes" />
    <input type="hidden" name="area" value="default" />
  </form>
</div>

            
          </div>

          
<div>
  <a style="color:#F05732" href="https://pytorch.org/docs/stable/_modules/torch/autograd/function.html">
    You are viewing unstable developer preview docs.
    Click here to view docs for latest stable release.
  </a>
</div>

            
            
              
            
            
              <p class="caption"><span class="caption-text">Notes</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../notes/amp_examples.html">Automatic Mixed Precision examples</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../notes/autograd.html">Autograd mechanics</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../notes/broadcasting.html">Broadcasting semantics</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../notes/cpu_threading_torchscript_inference.html">CPU threading and TorchScript inference</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../notes/cuda.html">CUDA semantics</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../notes/ddp.html">Distributed Data Parallel</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../notes/extending.html">Extending PyTorch</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../notes/faq.html">Frequently Asked Questions</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../notes/large_scale_deployments.html">Features for large-scale deployments</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../notes/multiprocessing.html">Multiprocessing best practices</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../notes/randomness.html">Reproducibility</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../notes/serialization.html">Serialization semantics</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../notes/windows.html">Windows FAQ</a></li>
</ul>
<p class="caption"><span class="caption-text">Language Bindings</span></p>
<ul>
<li class="toctree-l1"><a class="reference external" href="https://pytorch.org/cppdocs/">C++ API</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../packages.html">Javadoc</a></li>
</ul>
<p class="caption"><span class="caption-text">Python API</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../torch.html">torch</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../nn.html">torch.nn</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../nn.functional.html">torch.nn.functional</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../tensors.html">torch.Tensor</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../tensor_attributes.html">Tensor Attributes</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../tensor_view.html">Tensor Views</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../autograd.html">torch.autograd</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../cuda.html">torch.cuda</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../amp.html">torch.cuda.amp</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../distributed.html">torch.distributed</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../distributions.html">torch.distributions</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../hub.html">torch.hub</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../jit.html">torch.jit</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../nn.init.html">torch.nn.init</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../onnx.html">torch.onnx</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../optim.html">torch.optim</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../quantization.html">Quantization</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../rpc/index.html">Distributed RPC Framework</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../random.html">torch.random</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../sparse.html">torch.sparse</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../storage.html">torch.Storage</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../bottleneck.html">torch.utils.bottleneck</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../checkpoint.html">torch.utils.checkpoint</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../cpp_extension.html">torch.utils.cpp_extension</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../data.html">torch.utils.data</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dlpack.html">torch.utils.dlpack</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../model_zoo.html">torch.utils.model_zoo</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../tensorboard.html">torch.utils.tensorboard</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../type_info.html">Type Info</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../named_tensor.html">Named Tensors</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../name_inference.html">Named Tensors operator coverage</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../__config__.html">torch.__config__</a></li>
</ul>
<p class="caption"><span class="caption-text">Libraries</span></p>
<ul>
<li class="toctree-l1"><a class="reference external" href="https://pytorch.org/audio">torchaudio</a></li>
<li class="toctree-l1"><a class="reference external" href="https://pytorch.org/text">torchtext</a></li>
<li class="toctree-l1"><a class="reference external" href="https://pytorch.org/elastic/">TorchElastic</a></li>
<li class="toctree-l1"><a class="reference external" href="https://pytorch.org/serve">TorchServe</a></li>
<li class="toctree-l1"><a class="reference external" href="http://pytorch.org/xla/">PyTorch on XLA Devices</a></li>
</ul>
<p class="caption"><span class="caption-text">Community</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../community/contribution_guide.html">PyTorch Contribution Guide</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../community/governance.html">PyTorch Governance</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../community/persons_of_interest.html">PyTorch Governance | Persons of Interest</a></li>
</ul>

            
          

        </div>
      </div>
    </nav>

    <div class="pytorch-container">
      <div class="pytorch-page-level-bar" id="pytorch-page-level-bar">
        <div class="pytorch-breadcrumbs-wrapper">
          















<div role="navigation" aria-label="breadcrumbs navigation">

  <ul class="pytorch-breadcrumbs">
    
      <li>
        <a href="../../../index.html">
          
            Docs
          
        </a> &gt;
      </li>

        
          <li><a href="../../index.html">Module code</a> &gt;</li>
        
          <li><a href="../../torch.html">torch</a> &gt;</li>
        
          <li><a href="../autograd.html">torch.autograd</a> &gt;</li>
        
      <li>torch.autograd.function</li>
    
    
      <li class="pytorch-breadcrumbs-aside">
        
      </li>
    
  </ul>

  
</div>
        </div>

        <div class="pytorch-shortcuts-wrapper" id="pytorch-shortcuts-wrapper">
          Shortcuts
        </div>
      </div>

      <section data-toggle="wy-nav-shift" id="pytorch-content-wrap" class="pytorch-content-wrap">
        <div class="pytorch-content-left">

        
          
          <div class="rst-content">
          
            <div role="main" class="main-content" itemscope="itemscope" itemtype="http://schema.org/Article">
             <article itemprop="articleBody" id="pytorch-article" class="pytorch-article">
              
  <h1>Source code for torch.autograd.function</h1><div class="highlight"><pre>
<span></span><span class="kn">import</span> <span class="nn">torch</span>
<span class="kn">import</span> <span class="nn">torch._C</span> <span class="k">as</span> <span class="nn">_C</span>
<span class="kn">import</span> <span class="nn">torch.utils.hooks</span> <span class="k">as</span> <span class="nn">hooks</span>
<span class="kn">from</span> <span class="nn">torch._six</span> <span class="kn">import</span> <span class="n">with_metaclass</span>
<span class="kn">import</span> <span class="nn">functools</span>
<span class="kn">import</span> <span class="nn">warnings</span>
<span class="kn">from</span> <span class="nn">collections</span> <span class="kn">import</span> <span class="n">OrderedDict</span>


<div class="viewcode-block" id="_ContextMethodMixin"><a class="viewcode-back" href="../../../autograd.html#torch.autograd._ContextMethodMixin">[docs]</a><span class="k">class</span> <span class="nc">_ContextMethodMixin</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>

<div class="viewcode-block" id="_ContextMethodMixin.save_for_backward"><a class="viewcode-back" href="../../../autograd.html#torch.autograd._ContextMethodMixin.save_for_backward">[docs]</a>    <span class="k">def</span> <span class="nf">save_for_backward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">tensors</span><span class="p">):</span>
        <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Saves given tensors for a future call to :func:`~Function.backward`.</span>

<span class="sd">        **This should be called at most once, and only from inside the**</span>
<span class="sd">        :func:`forward` **method.**</span>

<span class="sd">        Later, saved tensors can be accessed through the :attr:`saved_tensors`</span>
<span class="sd">        attribute. Before returning them to the user, a check is made to ensure</span>
<span class="sd">        they weren&#39;t used in any in-place operation that modified their content.</span>

<span class="sd">        Arguments can also be ``None``.</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">to_save</span> <span class="o">=</span> <span class="n">tensors</span></div>

<div class="viewcode-block" id="_ContextMethodMixin.mark_dirty"><a class="viewcode-back" href="../../../autograd.html#torch.autograd._ContextMethodMixin.mark_dirty">[docs]</a>    <span class="k">def</span> <span class="nf">mark_dirty</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">):</span>
        <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Marks given tensors as modified in an in-place operation.</span>

<span class="sd">        **This should be called at most once, only from inside the**</span>
<span class="sd">        :func:`forward` **method, and all arguments should be inputs.**</span>

<span class="sd">        Every tensor that&#39;s been modified in-place in a call to :func:`forward`</span>
<span class="sd">        should be given to this function, to ensure correctness of our checks.</span>
<span class="sd">        It doesn&#39;t matter whether the function is called before or after</span>
<span class="sd">        modification.</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">dirty_tensors</span> <span class="o">=</span> <span class="n">args</span></div>

    <span class="k">def</span> <span class="nf">mark_shared_storage</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">pairs</span><span class="p">):</span>
        <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span>
            <span class="s1">&#39;mark_shared_storage is deprecated. &#39;</span>
            <span class="s1">&#39;Tensors with shared storages are automatically tracked. Note &#39;</span>
            <span class="s1">&#39;that calls to `set_()` are not tracked&#39;</span><span class="p">)</span>

<div class="viewcode-block" id="_ContextMethodMixin.mark_non_differentiable"><a class="viewcode-back" href="../../../autograd.html#torch.autograd._ContextMethodMixin.mark_non_differentiable">[docs]</a>    <span class="k">def</span> <span class="nf">mark_non_differentiable</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">):</span>
        <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Marks outputs as non-differentiable.</span>

<span class="sd">        **This should be called at most once, only from inside the**</span>
<span class="sd">        :func:`forward` **method, and all arguments should be outputs.**</span>

<span class="sd">        This will mark outputs as not requiring gradients, increasing the</span>
<span class="sd">        efficiency of backward computation. You still need to accept a gradient</span>
<span class="sd">        for each output in :meth:`~Function.backward`, but it&#39;s always going to</span>
<span class="sd">        be a zero tensor with the same shape as the shape of a corresponding</span>
<span class="sd">        output.</span>

<span class="sd">        This is used e.g. for indices returned from a max :class:`Function`.</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">non_differentiable</span> <span class="o">=</span> <span class="n">args</span></div></div>


<span class="k">class</span> <span class="nc">_HookMixin</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>

    <span class="nd">@staticmethod</span>
    <span class="k">def</span> <span class="nf">_register_hook</span><span class="p">(</span><span class="n">backward_hooks</span><span class="p">,</span> <span class="n">hook</span><span class="p">):</span>
        <span class="k">if</span> <span class="n">backward_hooks</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
            <span class="n">backward_hooks</span> <span class="o">=</span> <span class="n">OrderedDict</span><span class="p">()</span>
        <span class="n">handle</span> <span class="o">=</span> <span class="n">hooks</span><span class="o">.</span><span class="n">RemovableHandle</span><span class="p">(</span><span class="n">backward_hooks</span><span class="p">)</span>
        <span class="n">backward_hooks</span><span class="p">[</span><span class="n">handle</span><span class="o">.</span><span class="n">id</span><span class="p">]</span> <span class="o">=</span> <span class="n">hook</span>
        <span class="k">return</span> <span class="n">backward_hooks</span><span class="p">,</span> <span class="n">handle</span>


<span class="k">class</span> <span class="nc">BackwardCFunction</span><span class="p">(</span><span class="n">_C</span><span class="o">.</span><span class="n">_FunctionBase</span><span class="p">,</span> <span class="n">_ContextMethodMixin</span><span class="p">,</span> <span class="n">_HookMixin</span><span class="p">):</span>
    <span class="n">_is_legacy</span> <span class="o">=</span> <span class="kc">False</span>

    <span class="k">def</span> <span class="nf">apply</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">):</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_forward_cls</span><span class="o">.</span><span class="n">backward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">)</span>


<span class="k">class</span> <span class="nc">FunctionMeta</span><span class="p">(</span><span class="nb">type</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;Function metaclass.</span>

<span class="sd">    This metaclass sets up the following properties:</span>
<span class="sd">        _is_legacy: True if forward is not defined as a static method.</span>
<span class="sd">        _backward_cls: The Function class corresponding to the differentiated</span>
<span class="sd">            version of this function (which is generated on the fly by this</span>
<span class="sd">            metaclass).</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="n">bases</span><span class="p">,</span> <span class="n">attrs</span><span class="p">):</span>
        <span class="k">for</span> <span class="n">super_cls</span> <span class="ow">in</span> <span class="bp">cls</span><span class="o">.</span><span class="n">mro</span><span class="p">():</span>
            <span class="n">forward</span> <span class="o">=</span> <span class="n">super_cls</span><span class="o">.</span><span class="vm">__dict__</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;forward&#39;</span><span class="p">)</span>
            <span class="k">if</span> <span class="n">forward</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
                <span class="n">has_static_forward</span> <span class="o">=</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">forward</span><span class="p">,</span> <span class="nb">staticmethod</span><span class="p">)</span> <span class="ow">or</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">forward</span><span class="p">,</span> <span class="nb">classmethod</span><span class="p">)</span>
                <span class="k">break</span>

        <span class="bp">cls</span><span class="o">.</span><span class="n">_is_legacy</span> <span class="o">=</span> <span class="ow">not</span> <span class="n">has_static_forward</span>

        <span class="c1"># old-style functions</span>
        <span class="k">if</span> <span class="ow">not</span> <span class="n">has_static_forward</span><span class="p">:</span>
            <span class="k">return</span> <span class="nb">super</span><span class="p">(</span><span class="n">FunctionMeta</span><span class="p">,</span> <span class="bp">cls</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">bases</span><span class="p">,</span> <span class="n">attrs</span><span class="p">)</span>

        <span class="n">backward_fn</span> <span class="o">=</span> <span class="nb">type</span><span class="p">(</span><span class="n">name</span> <span class="o">+</span> <span class="s1">&#39;Backward&#39;</span><span class="p">,</span> <span class="p">(</span><span class="n">BackwardCFunction</span><span class="p">,),</span> <span class="p">{</span><span class="s1">&#39;_forward_cls&#39;</span><span class="p">:</span> <span class="bp">cls</span><span class="p">})</span>
        <span class="bp">cls</span><span class="o">.</span><span class="n">_backward_cls</span> <span class="o">=</span> <span class="n">backward_fn</span>

        <span class="k">return</span> <span class="nb">super</span><span class="p">(</span><span class="n">FunctionMeta</span><span class="p">,</span> <span class="bp">cls</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">bases</span><span class="p">,</span> <span class="n">attrs</span><span class="p">)</span>


<div class="viewcode-block" id="Function"><a class="viewcode-back" href="../../../autograd.html#torch.autograd.Function">[docs]</a><span class="k">class</span> <span class="nc">Function</span><span class="p">(</span><span class="n">with_metaclass</span><span class="p">(</span><span class="n">FunctionMeta</span><span class="p">,</span> <span class="n">_C</span><span class="o">.</span><span class="n">_FunctionBase</span><span class="p">,</span> <span class="n">_ContextMethodMixin</span><span class="p">,</span> <span class="n">_HookMixin</span><span class="p">)):</span>
    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Records operation history and defines formulas for differentiating ops.</span>

<span class="sd">    Every operation performed on :class:`Tensor` s creates a new function</span>
<span class="sd">    object, that performs the computation, and records that it happened.</span>
<span class="sd">    The history is retained in the form of a DAG of functions, with edges</span>
<span class="sd">    denoting data dependencies (``input &lt;- output``). Then, when backward is</span>
<span class="sd">    called, the graph is processed in the topological ordering, by calling</span>
<span class="sd">    :func:`backward` methods of each :class:`Function` object, and passing</span>
<span class="sd">    returned gradients on to next :class:`Function` s.</span>

<span class="sd">    Normally, the only way users interact with functions is by creating</span>
<span class="sd">    subclasses and defining new operations. This is a recommended way of</span>
<span class="sd">    extending torch.autograd.</span>

<span class="sd">    Examples::</span>

<span class="sd">        &gt;&gt;&gt; class Exp(Function):</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt;     @staticmethod</span>
<span class="sd">        &gt;&gt;&gt;     def forward(ctx, i):</span>
<span class="sd">        &gt;&gt;&gt;         result = i.exp()</span>
<span class="sd">        &gt;&gt;&gt;         ctx.save_for_backward(result)</span>
<span class="sd">        &gt;&gt;&gt;         return result</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt;     @staticmethod</span>
<span class="sd">        &gt;&gt;&gt;     def backward(ctx, grad_output):</span>
<span class="sd">        &gt;&gt;&gt;         result, = ctx.saved_tensors</span>
<span class="sd">        &gt;&gt;&gt;         return grad_output * result</span>
<span class="sd">        &gt;&gt;&gt;</span>
<span class="sd">        &gt;&gt;&gt; #Use it by calling the apply method:</span>
<span class="sd">        &gt;&gt;&gt; output = Exp.apply(input)</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="k">def</span> <span class="fm">__call__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
        <span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span>
            <span class="s2">&quot;Legacy autograd function with non-static forward method is deprecated. &quot;</span>
            <span class="s2">&quot;Please use new-style autograd function with static forward method. &quot;</span>
            <span class="s2">&quot;(Example: https://pytorch.org/docs/stable/autograd.html#torch.autograd.Function)&quot;</span><span class="p">)</span>

    <span class="c1"># for the tracer</span>
    <span class="n">is_traceable</span> <span class="o">=</span> <span class="kc">False</span>

<div class="viewcode-block" id="Function.forward"><a class="viewcode-back" href="../../../autograd.html#torch.autograd.Function.forward">[docs]</a>    <span class="nd">@staticmethod</span>
    <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="n">ctx</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
        <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Performs the operation.</span>

<span class="sd">        This function is to be overridden by all subclasses.</span>

<span class="sd">        It must accept a context ctx as the first argument, followed by any</span>
<span class="sd">        number of arguments (tensors or other types).</span>

<span class="sd">        The context can be used to store tensors that can be then retrieved</span>
<span class="sd">        during the backward pass.</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">(</span><span class="s2">&quot;You must implement the forward function for custom&quot;</span>
                                  <span class="s2">&quot; autograd.Function.&quot;</span><span class="p">)</span></div>

<div class="viewcode-block" id="Function.backward"><a class="viewcode-back" href="../../../autograd.html#torch.autograd.Function.backward">[docs]</a>    <span class="nd">@staticmethod</span>
    <span class="k">def</span> <span class="nf">backward</span><span class="p">(</span><span class="n">ctx</span><span class="p">,</span> <span class="o">*</span><span class="n">grad_outputs</span><span class="p">):</span>
        <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Defines a formula for differentiating the operation.</span>

<span class="sd">        This function is to be overridden by all subclasses.</span>

<span class="sd">        It must accept a context :attr:`ctx` as the first argument, followed by</span>
<span class="sd">        as many outputs did :func:`forward` return, and it should return as many</span>
<span class="sd">        tensors, as there were inputs to :func:`forward`. Each argument is the</span>
<span class="sd">        gradient w.r.t the given output, and each returned value should be the</span>
<span class="sd">        gradient w.r.t. the corresponding input.</span>

<span class="sd">        The context can be used to retrieve tensors saved during the forward</span>
<span class="sd">        pass. It also has an attribute :attr:`ctx.needs_input_grad` as a tuple</span>
<span class="sd">        of booleans representing whether each input needs gradient. E.g.,</span>
<span class="sd">        :func:`backward` will have ``ctx.needs_input_grad[0] = True`` if the</span>
<span class="sd">        first input to :func:`forward` needs gradient computated w.r.t. the</span>
<span class="sd">        output.</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">(</span><span class="s2">&quot;You must implement the backward function for custom&quot;</span>
                                  <span class="s2">&quot; autograd.Function.&quot;</span><span class="p">)</span></div></div>


<span class="k">def</span> <span class="nf">once_differentiable</span><span class="p">(</span><span class="n">fn</span><span class="p">):</span>

    <span class="nd">@functools</span><span class="o">.</span><span class="n">wraps</span><span class="p">(</span><span class="n">fn</span><span class="p">)</span>
    <span class="k">def</span> <span class="nf">wrapper</span><span class="p">(</span><span class="n">ctx</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">):</span>
        <span class="k">with</span> <span class="n">torch</span><span class="o">.</span><span class="n">no_grad</span><span class="p">():</span>
            <span class="n">outputs</span> <span class="o">=</span> <span class="n">fn</span><span class="p">(</span><span class="n">ctx</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">)</span>

        <span class="k">if</span> <span class="ow">not</span> <span class="n">torch</span><span class="o">.</span><span class="n">is_grad_enabled</span><span class="p">():</span>
            <span class="k">return</span> <span class="n">outputs</span>

        <span class="c1"># If any of the inputs have requires_grad=True, we force the outputs</span>
        <span class="c1"># to have requires_grad=True but point to a grad_fn which throws an</span>
        <span class="c1"># error message during (double) back-propagation.</span>
        <span class="c1"># XXX: this is only an approximation of requires_grad - there&#39;s no way</span>
        <span class="c1"># to figure out if fn didn&#39;t use ctx.saved_tensors and as a result</span>
        <span class="c1"># some Tensors might require grad, even if no args do.</span>
        <span class="c1"># Unfortunately, this leads to unexpected error messages (&quot;no nodes</span>
        <span class="c1"># require computing gradients&quot;), but I don&#39;t have a better idea.</span>
        <span class="c1"># These functions would raise an error in backward anyway.</span>
        <span class="n">requires_grad</span> <span class="o">=</span> <span class="nb">any</span><span class="p">(</span><span class="nb">isinstance</span><span class="p">(</span><span class="n">arg</span><span class="p">,</span> <span class="n">torch</span><span class="o">.</span><span class="n">Tensor</span><span class="p">)</span> <span class="ow">and</span> <span class="n">arg</span><span class="o">.</span><span class="n">requires_grad</span>
                            <span class="k">for</span> <span class="n">arg</span> <span class="ow">in</span> <span class="n">args</span><span class="p">)</span>
        <span class="k">if</span> <span class="ow">not</span> <span class="n">requires_grad</span><span class="p">:</span>
            <span class="k">return</span> <span class="n">outputs</span>

        <span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">outputs</span><span class="p">,</span> <span class="nb">tuple</span><span class="p">):</span>
            <span class="n">outputs</span> <span class="o">=</span> <span class="p">(</span><span class="n">outputs</span><span class="p">,)</span>

        <span class="n">err_fn</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">_C</span><span class="o">.</span><span class="n">_functions</span><span class="o">.</span><span class="n">DelayedError</span><span class="p">(</span>
            <span class="sa">b</span><span class="s2">&quot;trying to differentiate twice a function that was marked&quot;</span>
            <span class="sa">b</span><span class="s2">&quot;with @once_differentiable&quot;</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">outputs</span><span class="p">))</span>

        <span class="c1"># Create aliases of each output that has requires_grad=True. We need</span>
        <span class="c1"># at least one of the inputs to err_fn to require grad so that the</span>
        <span class="c1"># output will have a grad_fn.</span>
        <span class="k">def</span> <span class="nf">fake_requires_grad</span><span class="p">(</span><span class="n">var</span><span class="p">):</span>
            <span class="k">if</span> <span class="n">var</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
                <span class="n">var</span> <span class="o">=</span> <span class="n">var</span><span class="o">.</span><span class="n">detach</span><span class="p">()</span>
                <span class="n">var</span><span class="o">.</span><span class="n">requires_grad</span> <span class="o">=</span> <span class="kc">True</span>
            <span class="k">return</span> <span class="n">var</span>

        <span class="k">return</span> <span class="n">err_fn</span><span class="p">(</span><span class="o">*</span><span class="p">[</span><span class="n">fake_requires_grad</span><span class="p">(</span><span class="n">v</span><span class="p">)</span> <span class="k">for</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">outputs</span><span class="p">])</span>
    <span class="k">return</span> <span class="n">wrapper</span>


<span class="k">def</span> <span class="nf">traceable</span><span class="p">(</span><span class="n">fn_cls</span><span class="p">):</span>
    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Marks Function as traceable for the JIT.</span>

<span class="sd">    Traceable functions have additional restrictions - they can&#39;t pass any</span>
<span class="sd">    data-dependent values to backward (e.g. Prod passes the output, which makes</span>
<span class="sd">    it non-traceable), and their backward should be implemented entirely in terms</span>
<span class="sd">    of operations on autograd Tensors in all cases.</span>

<span class="sd">    DON&#39;T USE THIS DECORATOR. IT IS FOR INTERNAL USE ONLY AND SHOULD BE HANDLED WITH</span>
<span class="sd">    CARE (or can give incorrect results otherwise).</span>
<span class="sd">    &quot;&quot;&quot;</span>
    <span class="n">fn_cls</span><span class="o">.</span><span class="n">is_traceable</span> <span class="o">=</span> <span class="kc">True</span>
    <span class="k">return</span> <span class="n">fn_cls</span>


<span class="k">class</span> <span class="nc">InplaceFunction</span><span class="p">(</span><span class="n">Function</span><span class="p">):</span>

    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">inplace</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">InplaceFunction</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">inplace</span> <span class="o">=</span> <span class="n">inplace</span>


<span class="k">def</span> <span class="nf">_nested_map</span><span class="p">(</span><span class="n">condition</span><span class="p">,</span> <span class="n">fn</span><span class="p">,</span> <span class="n">condition_msg</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
    <span class="k">def</span> <span class="nf">_map</span><span class="p">(</span><span class="n">obj</span><span class="p">):</span>
        <span class="k">if</span> <span class="n">condition</span><span class="p">(</span><span class="n">obj</span><span class="p">):</span>
            <span class="k">return</span> <span class="n">fn</span><span class="p">(</span><span class="n">obj</span><span class="p">)</span>
        <span class="k">elif</span> <span class="n">obj</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
            <span class="k">return</span> <span class="kc">None</span>
        <span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="p">(</span><span class="nb">list</span><span class="p">,</span> <span class="nb">tuple</span><span class="p">)):</span>
            <span class="n">mapped</span> <span class="o">=</span> <span class="p">(</span><span class="n">_map</span><span class="p">(</span><span class="n">x</span><span class="p">)</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">obj</span><span class="p">)</span>
            <span class="k">if</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="s1">&#39;_fields&#39;</span><span class="p">):</span>
                <span class="c1"># obj is namedtuple</span>
                <span class="k">return</span> <span class="nb">type</span><span class="p">(</span><span class="n">obj</span><span class="p">)(</span><span class="o">*</span><span class="n">mapped</span><span class="p">)</span>
            <span class="k">return</span> <span class="nb">type</span><span class="p">(</span><span class="n">obj</span><span class="p">)(</span><span class="n">mapped</span><span class="p">)</span>
        <span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="nb">dict</span><span class="p">):</span>
            <span class="k">return</span> <span class="p">{</span><span class="n">x</span> <span class="p">:</span> <span class="n">_map</span><span class="p">(</span><span class="n">obj</span><span class="p">[</span><span class="n">x</span><span class="p">])</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">obj</span><span class="p">}</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;Auto nesting doesn&#39;t know how to process &quot;</span>
                             <span class="s2">&quot;an input object of type &quot;</span> <span class="o">+</span> <span class="n">torch</span><span class="o">.</span><span class="n">typename</span><span class="p">(</span><span class="n">obj</span><span class="p">)</span> <span class="o">+</span>
                             <span class="p">(</span><span class="s2">&quot;. Accepted types: &quot;</span> <span class="o">+</span> <span class="n">condition_msg</span> <span class="o">+</span>
                              <span class="s2">&quot;, or lists/tuples of them&quot;</span>
                              <span class="k">if</span> <span class="n">condition_msg</span> <span class="k">else</span> <span class="s2">&quot;&quot;</span><span class="p">))</span>

    <span class="k">return</span> <span class="n">_map</span>


<span class="k">def</span> <span class="nf">_jit_unwrap_structured</span><span class="p">(</span><span class="n">obj</span><span class="p">):</span>
    <span class="k">if</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="s2">&quot;_jit_unwrap&quot;</span><span class="p">):</span>
        <span class="k">return</span> <span class="n">obj</span><span class="o">.</span><span class="n">_jit_unwrap</span><span class="p">()</span>
    <span class="k">return</span> <span class="n">obj</span>


<span class="k">def</span> <span class="nf">_iter_filter</span><span class="p">(</span><span class="n">condition</span><span class="p">,</span> <span class="n">allow_unknown</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">condition_msg</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
                 <span class="n">conversion</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
    <span class="k">def</span> <span class="nf">_iter</span><span class="p">(</span><span class="n">obj</span><span class="p">):</span>
        <span class="k">if</span> <span class="n">conversion</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
            <span class="n">obj</span> <span class="o">=</span> <span class="n">conversion</span><span class="p">(</span><span class="n">obj</span><span class="p">)</span>
        <span class="k">if</span> <span class="n">condition</span><span class="p">(</span><span class="n">obj</span><span class="p">):</span>
            <span class="k">yield</span> <span class="n">obj</span>
        <span class="k">elif</span> <span class="n">obj</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
            <span class="k">return</span>
        <span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="p">(</span><span class="nb">list</span><span class="p">,</span> <span class="nb">tuple</span><span class="p">)):</span>
            <span class="k">for</span> <span class="n">o</span> <span class="ow">in</span> <span class="n">obj</span><span class="p">:</span>
                <span class="k">for</span> <span class="n">var</span> <span class="ow">in</span> <span class="n">_iter</span><span class="p">(</span><span class="n">o</span><span class="p">):</span>
                    <span class="k">yield</span> <span class="n">var</span>
        <span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="nb">dict</span><span class="p">):</span>
            <span class="c1"># We only accept primitive key types, so we needn&#39;t inspect them</span>
            <span class="k">for</span> <span class="n">o</span> <span class="ow">in</span> <span class="n">obj</span><span class="o">.</span><span class="n">values</span><span class="p">():</span>
                <span class="k">for</span> <span class="n">var</span> <span class="ow">in</span> <span class="n">_iter</span><span class="p">(</span><span class="n">o</span><span class="p">):</span>
                    <span class="k">yield</span> <span class="n">var</span>
        <span class="k">elif</span> <span class="n">allow_unknown</span><span class="p">:</span>
            <span class="k">yield</span> <span class="n">obj</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;Auto nesting doesn&#39;t know how to process &quot;</span>
                             <span class="s2">&quot;an input object of type &quot;</span> <span class="o">+</span> <span class="n">torch</span><span class="o">.</span><span class="n">typename</span><span class="p">(</span><span class="n">obj</span><span class="p">)</span> <span class="o">+</span>
                             <span class="p">(</span><span class="s2">&quot;. Accepted types: &quot;</span> <span class="o">+</span> <span class="n">condition_msg</span> <span class="o">+</span>
                              <span class="s2">&quot;, or lists/tuples of them&quot;</span>
                              <span class="k">if</span> <span class="n">condition_msg</span> <span class="k">else</span> <span class="s2">&quot;&quot;</span><span class="p">))</span>

    <span class="k">return</span> <span class="n">_iter</span>


<span class="k">def</span> <span class="nf">_unflatten</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">proto</span><span class="p">):</span>
    <span class="c1"># unflatten a list or tuple input into a nested list/tuple structure</span>
    <span class="c1"># specified by proto</span>
    <span class="k">def</span> <span class="nf">unflatten_helper</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">proto</span><span class="p">):</span>
        <span class="n">res</span> <span class="o">=</span> <span class="p">[]</span>
        <span class="k">if</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">proto</span><span class="p">,</span> <span class="s2">&quot;_jit_wrap&quot;</span><span class="p">):</span>
            <span class="k">return</span> <span class="n">proto</span><span class="o">.</span><span class="n">_jit_wrap</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
        <span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">proto</span><span class="p">,</span> <span class="p">(</span><span class="nb">list</span><span class="p">,</span> <span class="nb">tuple</span><span class="p">)):</span>
            <span class="k">return</span> <span class="nb">input</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="nb">input</span><span class="p">[</span><span class="mi">1</span><span class="p">:]</span>
        <span class="k">for</span> <span class="n">e</span> <span class="ow">in</span> <span class="n">proto</span><span class="p">:</span>
            <span class="k">if</span> <span class="n">e</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
                <span class="n">res</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">e</span><span class="p">)</span>
            <span class="k">else</span><span class="p">:</span>
                <span class="n">res_e</span><span class="p">,</span> <span class="nb">input</span> <span class="o">=</span> <span class="n">unflatten_helper</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">e</span><span class="p">)</span>
                <span class="n">res</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">res_e</span><span class="p">)</span>
        <span class="k">return</span> <span class="nb">type</span><span class="p">(</span><span class="n">proto</span><span class="p">)(</span><span class="n">res</span><span class="p">),</span> <span class="nb">input</span>

    <span class="k">return</span> <span class="n">unflatten_helper</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">proto</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>


<span class="n">_iter_jit_values</span> <span class="o">=</span> <span class="n">_iter_filter</span><span class="p">(</span><span class="k">lambda</span> <span class="n">o</span><span class="p">:</span> <span class="n">o</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">or</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">o</span><span class="p">,</span> <span class="n">torch</span><span class="o">.</span><span class="n">_C</span><span class="o">.</span><span class="n">Value</span><span class="p">),</span>
                                <span class="n">condition_msg</span><span class="o">=</span><span class="s2">&quot;jit&#39;s Values or None&quot;</span><span class="p">)</span>
<span class="n">_iter_tensors</span> <span class="o">=</span> <span class="n">_iter_filter</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">torch</span><span class="o">.</span><span class="n">Tensor</span><span class="p">),</span> <span class="n">condition_msg</span><span class="o">=</span><span class="s2">&quot;Tensors&quot;</span><span class="p">,</span>
                             <span class="n">conversion</span><span class="o">=</span><span class="n">_jit_unwrap_structured</span><span class="p">)</span>
<span class="n">_iter_tensors_permissive</span> <span class="o">=</span> <span class="n">_iter_filter</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">torch</span><span class="o">.</span><span class="n">Tensor</span><span class="p">),</span>
                                        <span class="n">allow_unknown</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
                                        <span class="n">condition_msg</span><span class="o">=</span><span class="s2">&quot;Tensors (permissive)&quot;</span><span class="p">)</span>
<span class="n">_iter_None_tensors</span> <span class="o">=</span> <span class="n">_iter_filter</span><span class="p">(</span><span class="k">lambda</span> <span class="n">o</span><span class="p">:</span> <span class="n">o</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">or</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">o</span><span class="p">,</span> <span class="n">torch</span><span class="o">.</span><span class="n">Tensor</span><span class="p">),</span>
                                  <span class="n">condition_msg</span><span class="o">=</span><span class="s2">&quot;Tensors or None&quot;</span><span class="p">)</span>
<span class="n">_map_tensor_data</span> <span class="o">=</span> <span class="n">_nested_map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">torch</span><span class="o">.</span><span class="n">Tensor</span><span class="p">),</span> <span class="k">lambda</span> <span class="n">o</span><span class="p">:</span> <span class="n">o</span><span class="o">.</span><span class="n">data</span><span class="p">,</span>
                               <span class="n">condition_msg</span><span class="o">=</span><span class="s2">&quot;Tensors&quot;</span><span class="p">)</span>


<span class="k">class</span> <span class="nc">NestedIOFunction</span><span class="p">(</span><span class="n">Function</span><span class="p">):</span>

    <span class="k">def</span> <span class="nf">_do_forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="nb">input</span><span class="p">):</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_nested_input</span> <span class="o">=</span> <span class="nb">input</span>
        <span class="n">flat_input</span> <span class="o">=</span> <span class="nb">tuple</span><span class="p">(</span><span class="n">_iter_tensors</span><span class="p">(</span><span class="nb">input</span><span class="p">))</span>
        <span class="n">flat_output</span> <span class="o">=</span> <span class="nb">super</span><span class="p">(</span><span class="n">NestedIOFunction</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">_do_forward</span><span class="p">(</span><span class="o">*</span><span class="n">flat_input</span><span class="p">)</span>
        <span class="n">nested_output</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_nested_output</span>
        <span class="n">nested_tensors</span> <span class="o">=</span> <span class="n">_unflatten</span><span class="p">(</span><span class="n">flat_output</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_nested_output</span><span class="p">)</span>
        <span class="k">return</span> <span class="n">nested_tensors</span>

    <span class="k">def</span> <span class="nf">_do_backward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">gradients</span><span class="p">,</span> <span class="n">retain_variables</span><span class="p">):</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">retain_variables</span> <span class="o">=</span> <span class="n">retain_variables</span>
        <span class="n">result</span> <span class="o">=</span> <span class="nb">super</span><span class="p">(</span><span class="n">NestedIOFunction</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">_do_backward</span><span class="p">(</span><span class="n">gradients</span><span class="p">,</span> <span class="n">retain_variables</span><span class="p">)</span>
        <span class="k">if</span> <span class="ow">not</span> <span class="n">retain_variables</span><span class="p">:</span>
            <span class="k">del</span> <span class="bp">self</span><span class="o">.</span><span class="n">_nested_output</span>
            <span class="k">del</span> <span class="bp">self</span><span class="o">.</span><span class="n">_to_save_nested</span>
        <span class="k">return</span> <span class="n">result</span>

    <span class="k">def</span> <span class="nf">backward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">gradients</span><span class="p">):</span>
        <span class="n">nested_gradients</span> <span class="o">=</span> <span class="n">_unflatten</span><span class="p">(</span><span class="n">gradients</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_nested_output</span><span class="p">)</span>
        <span class="n">result</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">backward_extended</span><span class="p">(</span><span class="o">*</span><span class="n">nested_gradients</span><span class="p">)</span>
        <span class="k">return</span> <span class="nb">tuple</span><span class="p">(</span><span class="n">_iter_None_tensors</span><span class="p">(</span><span class="n">result</span><span class="p">))</span>

    <span class="fm">__call__</span> <span class="o">=</span> <span class="n">_do_forward</span>

    <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">):</span>
        <span class="n">nested_tensors</span> <span class="o">=</span> <span class="n">_map_tensor_data</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_nested_input</span><span class="p">)</span>
        <span class="n">result</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">forward_extended</span><span class="p">(</span><span class="o">*</span><span class="n">nested_tensors</span><span class="p">)</span>
        <span class="k">del</span> <span class="bp">self</span><span class="o">.</span><span class="n">_nested_input</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_nested_output</span> <span class="o">=</span> <span class="n">result</span>
        <span class="k">return</span> <span class="nb">tuple</span><span class="p">(</span><span class="n">_iter_tensors</span><span class="p">(</span><span class="n">result</span><span class="p">))</span>

    <span class="k">def</span> <span class="nf">save_for_backward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">):</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">to_save</span> <span class="o">=</span> <span class="nb">tuple</span><span class="p">(</span><span class="n">_iter_tensors</span><span class="p">(</span><span class="n">args</span><span class="p">))</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_to_save_nested</span> <span class="o">=</span> <span class="n">args</span>

    <span class="nd">@property</span>
    <span class="k">def</span> <span class="nf">saved_tensors</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="n">flat_tensors</span> <span class="o">=</span> <span class="nb">super</span><span class="p">(</span><span class="n">NestedIOFunction</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">saved_tensors</span>
        <span class="k">return</span> <span class="n">_unflatten</span><span class="p">(</span><span class="n">flat_tensors</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_to_save_nested</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">mark_dirty</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">dirty_tensors</span> <span class="o">=</span> <span class="nb">tuple</span><span class="p">(</span><span class="n">_iter_tensors</span><span class="p">((</span><span class="n">args</span><span class="p">,</span> <span class="n">kwargs</span><span class="p">)))</span>

    <span class="k">def</span> <span class="nf">mark_non_differentiable</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">non_differentiable</span> <span class="o">=</span> <span class="nb">tuple</span><span class="p">(</span><span class="n">_iter_tensors</span><span class="p">((</span><span class="n">args</span><span class="p">,</span> <span class="n">kwargs</span><span class="p">)))</span>

    <span class="k">def</span> <span class="nf">forward_extended</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="nb">input</span><span class="p">):</span>
        <span class="k">raise</span> <span class="ne">NotImplementedError</span>

    <span class="k">def</span> <span class="nf">backward_extended</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">grad_output</span><span class="p">):</span>
        <span class="k">raise</span> <span class="ne">NotImplementedError</span>
</pre></div>

             </article>
             
            </div>
            <footer>
  

  

    <hr>

  

  <div role="contentinfo">
    <p>
        &copy; Copyright 2019, Torch Contributors.

    </p>
  </div>
    
      <div>
        Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
      </div>
     

</footer>

          </div>
        </div>

        <div class="pytorch-content-right" id="pytorch-content-right">
          <div class="pytorch-right-menu" id="pytorch-right-menu">
            <div class="pytorch-side-scroll" id="pytorch-side-scroll-right">
              
            </div>
          </div>
        </div>
      </section>
    </div>

  


  

     
       <script type="text/javascript" id="documentation_options" data-url_root="../../../" src="../../../_static/documentation_options.js"></script>
         <script src="../../../_static/jquery.js"></script>
         <script src="../../../_static/underscore.js"></script>
         <script src="../../../_static/doctools.js"></script>
         <script src="../../../_static/language_data.js"></script>
     

  

  <script type="text/javascript" src="../../../_static/js/vendor/popper.min.js"></script>
  <script type="text/javascript" src="../../../_static/js/vendor/bootstrap.min.js"></script>
  <script src="https://cdnjs.cloudflare.com/ajax/libs/list.js/1.5.0/list.min.js"></script>
  <script type="text/javascript" src="../../../_static/js/theme.js"></script>

  <script type="text/javascript">
      jQuery(function () {
          SphinxRtdTheme.Navigation.enable(true);
      });
  </script>
 
<script>
  (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
  (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
  m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
  })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');

  ga('create', 'UA-90545585-1', 'auto');
  ga('send', 'pageview');

</script>

<script async src="https://www.googletagmanager.com/gtag/js?id=UA-117752657-2"></script>

<script>
  window.dataLayer = window.dataLayer || [];

  function gtag(){dataLayer.push(arguments);}

  gtag('js', new Date());
  gtag('config', 'UA-117752657-2');
</script>

<img height="1" width="1" style="border-style:none;" alt="" src="https://www.googleadservices.com/pagead/conversion/795629140/?label=txkmCPmdtosBENSssfsC&amp;guid=ON&amp;script=0"/>


  <!-- Begin Footer -->

  <div class="container-fluid docs-tutorials-resources" id="docs-tutorials-resources">
    <div class="container">
      <div class="row">
        <div class="col-md-4 text-center">
          <h2>Docs</h2>
          <p>Access comprehensive developer documentation for PyTorch</p>
          <a class="with-right-arrow" href="https://pytorch.org/docs/stable/index.html">View Docs</a>
        </div>

        <div class="col-md-4 text-center">
          <h2>Tutorials</h2>
          <p>Get in-depth tutorials for beginners and advanced developers</p>
          <a class="with-right-arrow" href="https://pytorch.org/tutorials">View Tutorials</a>
        </div>

        <div class="col-md-4 text-center">
          <h2>Resources</h2>
          <p>Find development resources and get your questions answered</p>
          <a class="with-right-arrow" href="https://pytorch.org/resources">View Resources</a>
        </div>
      </div>
    </div>
  </div>

  <footer class="site-footer">
    <div class="container footer-container">
      <div class="footer-logo-wrapper">
        <a href="https://pytorch.org/" class="footer-logo"></a>
      </div>

      <div class="footer-links-wrapper">
        <div class="footer-links-col">
          <ul>
            <li class="list-title"><a href="https://pytorch.org/">PyTorch</a></li>
            <li><a href="https://pytorch.org/get-started">Get Started</a></li>
            <li><a href="https://pytorch.org/features">Features</a></li>
            <li><a href="https://pytorch.org/ecosystem">Ecosystem</a></li>
            <li><a href="https://pytorch.org/blog/">Blog</a></li>
            <li><a href="https://github.com/pytorch/pytorch/blob/master/CONTRIBUTING.md">Contributing</a></li>
          </ul>
        </div>

        <div class="footer-links-col">
          <ul>
            <li class="list-title"><a href="https://pytorch.org/resources">Resources</a></li>
            <li><a href="https://pytorch.org/tutorials">Tutorials</a></li>
            <li><a href="https://pytorch.org/docs/stable/index.html">Docs</a></li>
            <li><a href="https://discuss.pytorch.org" target="_blank">Discuss</a></li>
            <li><a href="https://github.com/pytorch/pytorch/issues" target="_blank">Github Issues</a></li>
            <li><a href="https://pytorch.org/assets/brand-guidelines/PyTorch-Brand-Guidelines.pdf" target="_blank">Brand Guidelines</a></li>
          </ul>
        </div>

        <div class="footer-links-col follow-us-col">
          <ul>
            <li class="list-title">Stay Connected</li>
            <li>
              <div id="mc_embed_signup">
                <form
                  action="https://twitter.us14.list-manage.com/subscribe/post?u=75419c71fe0a935e53dfa4a3f&id=91d0dccd39"
                  method="post"
                  id="mc-embedded-subscribe-form"
                  name="mc-embedded-subscribe-form"
                  class="email-subscribe-form validate"
                  target="_blank"
                  novalidate>
                  <div id="mc_embed_signup_scroll" class="email-subscribe-form-fields-wrapper">
                    <div class="mc-field-group">
                      <label for="mce-EMAIL" style="display:none;">Email Address</label>
                      <input type="email" value="" name="EMAIL" class="required email" id="mce-EMAIL" placeholder="Email Address">
                    </div>

                    <div id="mce-responses" class="clear">
                      <div class="response" id="mce-error-response" style="display:none"></div>
                      <div class="response" id="mce-success-response" style="display:none"></div>
                    </div>    <!-- real people should not fill this in and expect good things - do not remove this or risk form bot signups-->

                    <div style="position: absolute; left: -5000px;" aria-hidden="true"><input type="text" name="b_75419c71fe0a935e53dfa4a3f_91d0dccd39" tabindex="-1" value=""></div>

                    <div class="clear">
                      <input type="submit" value="" name="subscribe" id="mc-embedded-subscribe" class="button email-subscribe-button">
                    </div>
                  </div>
                </form>
              </div>

            </li>
          </ul>

          <div class="footer-social-icons">
            <a href="https://www.facebook.com/pytorch" target="_blank" class="facebook"></a>
            <a href="https://twitter.com/pytorch" target="_blank" class="twitter"></a>
            <a href="https://www.youtube.com/pytorch" target="_blank" class="youtube"></a>
          </div>
        </div>
      </div>
    </div>
  </footer>

  <div class="cookie-banner-wrapper">
  <div class="container">
    <p class="gdpr-notice">To analyze traffic and optimize your experience, we serve cookies on this site. By clicking or navigating, you agree to allow our usage of cookies. As the current maintainers of this site, Facebook’s Cookies Policy applies. Learn more, including about available controls: <a href="https://www.facebook.com/policies/cookies/">Cookies Policy</a>.</p>
    <img class="close-button" src="../../../_static/images/pytorch-x.svg">
  </div>
</div>

  <!-- End Footer -->

  <!-- Begin Mobile Menu -->

  <div class="mobile-main-menu">
    <div class="container-fluid">
      <div class="container">
        <div class="mobile-main-menu-header-container">
          <a class="header-logo" href="https://pytorch.org/" aria-label="PyTorch"></a>
          <a class="main-menu-close-button" href="#" data-behavior="close-mobile-menu"></a>
        </div>
      </div>
    </div>

    <div class="mobile-main-menu-links-container">
      <div class="main-menu">
        <ul>
          <li>
            <a href="https://pytorch.org/get-started">Get Started</a>
          </li>

          <li>
            <a href="https://pytorch.org/features">Features</a>
          </li>

          <li>
            <a href="https://pytorch.org/ecosystem">Ecosystem</a>
          </li>

          <li>
            <a href="https://pytorch.org/mobile">Mobile</a>
          </li>

          <li>
            <a href="https://pytorch.org/hub">PyTorch Hub</a>
          </li>

          <li>
            <a href="https://pytorch.org/blog/">Blog</a>
          </li>

          <li>
            <a href="https://pytorch.org/tutorials">Tutorials</a>
          </li>

          <li class="active">
            <a href="https://pytorch.org/docs/stable/index.html">Docs</a>
          </li>

          <li>
            <a href="https://pytorch.org/resources">Resources</a>
          </li>

          <li>
            <a href="https://github.com/pytorch/pytorch">Github</a>
          </li>
        </ul>
      </div>
    </div>
  </div>

  <!-- End Mobile Menu -->

  <script type="text/javascript" src="../../../_static/js/vendor/anchor.min.js"></script>

  <script type="text/javascript">
    $(document).ready(function() {
      mobileMenu.bind();
      mobileTOC.bind();
      pytorchAnchors.bind();
      sideMenus.bind();
      scrollToAnchor.bind();
      highlightNavigation.bind();
      mainMenuDropdown.bind();
      filterTags.bind();

      // Remove any empty p tags that Sphinx adds
      $("[data-tags='null']").remove();

      // Add class to links that have code blocks, since we cannot create links in code blocks
      $("article.pytorch-article a span.pre").each(function(e) {
        $(this).closest("a").addClass("has-code");
      });
    })
  </script>
</body>
</html>